General Notes on Meta Analyses

  • A Meta-analysis is the statistical combination of results from two or more separate studies.
  • Potential advantages of meta-analyses include:
    • Improved precision. Many studies are too small to provide convincing evidence about intervention effects in isolation. Estimation is usually improved when it is based on more information.
    • Answering questions not posed by individual studies. Primary studies often involve a specific type of participant and explicitly defined interventions. A selection of studies in which these characteristics differ can allow investigation of the consistency of effect across a wider range of populations and interventions. It may also, if relevant, allow reasons for differences in effect estimates to be investigated.
    • Settlement of controversies arising from apparently conflicting studies or to generate new hypotheses. Statistical synthesis of findings allows the degree of conflict to be formally assessed, and reasons for different results to be explored and quantified.

General Descriptives

0. General Info

# Number of unique articles
length(unique(meta_data$authors_short))
## [1] 43
# Number of effect sizes
dim(meta_data)
## [1] 211 159
# Meta sample size
sum(meta_data$n)
## [1] 121215
summary(meta_data$n)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    25.0    92.0   171.0   574.5   393.5 20796.0
# Average group size human condition
summary(meta_data$focus_n1)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    25.0    61.0    88.0   253.1   194.0 10398.0
# Average group size machine condition
summary(meta_data$focus_n2)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    25.0    62.0    88.0   253.5   194.0 10398.0

1. Publication Details

Number of Effect Sizes by Authors

plot_by_authors

Number of Effect Sizes by Outlets

plot_by_outlets

Number of Effect Sizes by Discipline

discipline_plotdata
## # A tibble: 8 x 4
##   discipline              n    pct lbl   
##   <fct>               <int>  <dbl> <chr> 
## 1 Information Systems    55 0.261  26.07%
## 2 Management             48 0.227  22.75%
## 3 Psychology             43 0.204  20.38%
## 4 Marketing              34 0.161  16.11%
## 5 Medicine               11 0.0521 5.21% 
## 6 General Interest       10 0.0474 4.74% 
## 7 Transportation          6 0.0284 2.84% 
## 8 Law                     4 0.0190 1.90%
plot_by_discipline

Number of Effect Sizes by Outlets and Discipline

disciplineou_plotdata
## # A tibble: 33 x 5
## # Groups:   discipline [8]
##    discipline       outlet                                         n   pct lbl  
##    <fct>            <fct>                                      <int> <dbl> <chr>
##  1 Law              Journal of Crime and Justice                   4 1     100% 
##  2 Transportation   Accident Analysis & Prevention                 6 1     100% 
##  3 Medicine         Medical Decision Making                        9 0.818 82%  
##  4 Psychology       Journal of Behavioral Decision Making         24 0.558 55.8%
##  5 General Interest Nature Human Behavior                          4 0.4   40%  
##  6 Marketing        Journal of Consumer Research                  12 0.353 35.3%
##  7 Management       International Journal of Selection and As~    15 0.312 31.2%
##  8 General Interest Heliyon                                        3 0.3   30%  
##  9 General Interest PLOS ONE                                       3 0.3   30%  
## 10 Psychology       Cognition                                     10 0.233 23.3%
## # ... with 23 more rows
plot_by_disciplineou

Number of Effect Sizes by Publication Year

pubyear_plotdata
## # A tibble: 15 x 4
##    pubyear     n     pct lbl   
##    <fct>   <int>   <dbl> <chr> 
##  1 2022       10 0.0474  4.74% 
##  2 2021       51 0.242   24.17%
##  3 2020       24 0.114   11.37%
##  4 2019       48 0.227   22.75%
##  5 2018       21 0.0995  9.95% 
##  6 2017        9 0.0427  4.27% 
##  7 2015        3 0.0142  1.42% 
##  8 2013        2 0.00948 0.95% 
##  9 2012       12 0.0569  5.69% 
## 10 2011       15 0.0711  7.11% 
## 11 2009        3 0.0142  1.42% 
## 12 2007        6 0.0284  2.84% 
## 13 2006        4 0.0190  1.90% 
## 14 2005        1 0.00474 0.47% 
## 15 2002        2 0.00948 0.95%
plot_by_pubyear

Number of Effect Sizes by Publication Type

pubtype_plotdata
## # A tibble: 3 x 4
##   pubtype                    n    pct lbl  
##   <fct>                  <int>  <dbl> <chr>
## 1 Journal article          186 0.882  88.2%
## 2 Conference proceedings    14 0.0664 6.6% 
## 3 Working Paper             11 0.0521 5.2%
plot_by_pubtype

Number of Effect Sizes by Publication Status

published_plotdata
## # A tibble: 2 x 4
##   published     n   pct lbl  
##   <fct>     <int> <dbl> <chr>
## 1 yes         186 0.882 88%  
## 2 no           25 0.118 12%
plot_by_published

Aversion: No (0) versus Yes (1)

aversion_plotdata
## # A tibble: 2 x 4
##   aversion     n   pct lbl  
##   <fct>    <int> <dbl> <chr>
## 1 1          145 0.687 69%  
## 2 0           66 0.313 31%
plot_by_aversion

3. Method Details

do.call("grid.arrange", c(plist_1, ncol = 2))

4. Sample Details

Sample Size

summary(meta_data$n)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    25.0    92.0   171.0   574.5   393.5 20796.0
hist(meta_data$n, main="Histogram for Sample Sizes", xlab="Sample Sizes")

Age

summary(meta_data$m_age)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.75   32.25   36.34   35.17   39.58   50.90      32
hist(meta_data$m_age, main="Histogram for Mean Ages", xlab="Mean Ages")

Percentage Females

summary(meta_data$percntg_females)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   22.60   45.00   49.00   51.89   57.00  100.00      18
hist(meta_data$percntg_females, main="Histogram for Percentages Females", xlab="Percentages Females")

Others

do.call("grid.arrange", c(plist_2, ncol = 2))

5. Domain Details

All Domains

domain_plotdata
## # A tibble: 15 x 4
##    domain                 n     pct lbl   
##    <fct>              <int>   <dbl> <chr> 
##  1 management            64 0.303   30.33%
##  2 medicine              41 0.194   19.43%
##  3 multiple              34 0.161   16.11%
##  4 law                   13 0.0616  6.16% 
##  5 finance                9 0.0427  4.27% 
##  6 shopping               9 0.0427  4.27% 
##  7 social                 9 0.0427  4.27% 
##  8 other                  8 0.0379  3.79% 
##  9 autonomous driving     7 0.0332  3.32% 
## 10 airport security       3 0.0142  1.42% 
## 11 art                    3 0.0142  1.42% 
## 12 baking/cooking         3 0.0142  1.42% 
## 13 military               3 0.0142  1.42% 
## 14 news                   3 0.0142  1.42% 
## 15 real estate            2 0.00948 0.95%
plot_by_domain

Management

management_plotdata
## # A tibble: 3 x 4
##   management     n    pct lbl  
##   <fct>      <int>  <dbl> <chr>
## 1 no           130 0.616  62%  
## 2 yes           75 0.355  36%  
## 3 <NA>           6 0.0284 3%
plot_by_management

Subjective versus Objective

subjobj_plotdata
## # A tibble: 3 x 4
##   subj_vs_obj     n   pct lbl  
##   <fct>       <int> <dbl> <chr>
## 1 objective     153 0.725 72.5%
## 2 subjective     34 0.161 16.1%
## 3 <NA>           24 0.114 11.4%
plot_by_subjobj

Severity

severity_plotdata
## # A tibble: 4 x 4
##   severity     n   pct lbl  
##   <fct>    <int> <dbl> <chr>
## 1 medium      77 0.365 36.5%
## 2 high        61 0.289 28.9%
## 3 low         38 0.180 18.0%
## 4 <NA>        35 0.166 16.6%
plot_by_severity

6. Stimuli Details

do.call("grid.arrange", c(plist_3, ncol = 2))

Moderator Descriptives

1. Moderators by Aversion (Binary) Plots

  • Tammo Bijmolt (in his slides) on the question of sufficient number of observations per moderator level: “Many issues have been used just a few times. These moderators (levels) cannot be studied. Solution: Drop the moderator or combine levels. Future research in the Discussion section..”
  • Moderators with < 30 observation in one level:
    • unpublished versus published work –> own limitation?
    • field study –> observation in discussion section?
    • discrete dv scaling –> probably strong correlation with dv measurement type = behavioral, there enough obs
    • incentive compatibility –> observation in discussion section?

Aversion by Publication Details

iplot_outlet_plot

iplot_discipline_plot

plist_ipd
## [[1]]

## 
## [[2]]

aversion_by_year_plot

Aversion by Method Details

plist_imd
## [[1]]

## 
## [[2]]

## 
## [[3]]

## 
## [[4]]

## 
## [[5]]

## 
## [[6]]

## 
## [[7]]

## 
## [[8]]

## 
## [[9]]

Aversion by Sample Details

plist_isd
## [[1]]

## 
## [[2]]

## 
## [[3]]

## 
## [[4]]

## 
## [[5]]

## 
## [[6]]

## 
## [[7]]

Aversion by Domain

plist_id
## [[1]]

## 
## [[2]]

## 
## [[3]]

## 
## [[4]]

Aversion by Stimuli Details

plist_istd
## [[1]]

## 
## [[2]]

## 
## [[3]]

## 
## [[4]]

## 
## [[5]]

2. Moderators by Cohens d Plots

Main findings in this section (Model Free Evidence)

  • Positive avg. cohen’s d when
    • outcome was measured on a discrete scale
    • when the task was shopping
    • when the task is social
    • when the task is news
  • Aversion significantly reduced, when:
    • DV was measured behaviorally (compared to self-report)
    • The task was related to shopping rather than military
    • The task is subjective, rather than objective
    • human condition involves no experts versus experts (i.e., When the human is not an expert (either explicitly described with the term “expert” or described as a professional, e.g., “a doctor”) aversion is significantly smaller)
    • machine description contains the term artificial intelligence compared to no mention (i.e., when the term artificial intelligence is present, aversion is significantly smaller)
    • sample was based in America
print(plist[1])
## $outlet

print(boxplotlist[1])
## $outlet

print(plist[2])
## $discipline

print(boxplotlist[2])
## $discipline

print(meanlist[1])
## $discipline
## # A tibble: 8 x 2
##   `modsdata[, i]`       name
##   <fct>                <dbl>
## 1 General Interest    -0.179
## 2 Information Systems -0.247
## 3 Law                 -0.107
## 4 Management          -0.106
## 5 Marketing           -0.373
## 6 Medicine            -0.170
## 7 Psychology          -0.398
## 8 Transportation      -0.631
print(aovlist[1])
## $discipline
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   7   3.45  0.4934   1.434  0.193
## Residuals     203  69.83  0.3440
TukeyHSD(aov(cohens_d ~ discipline, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ discipline, data = meta_data)
## 
## $discipline
##                                              diff        lwr       upr
## Information Systems-General Interest -0.067821818 -0.6853735 0.5497298
## Law-General Interest                  0.071575000 -0.9911771 1.1343271
## Management-General Interest           0.072650000 -0.5517909 0.6970909
## Marketing-General Interest           -0.193817647 -0.8400443 0.4524090
## Medicine-General Interest             0.008490909 -0.7764035 0.7933854
## Psychology-General Interest          -0.218813953 -0.8494832 0.4118553
## Transportation-General Interest      -0.451816667 -1.3794628 0.4758294
## Law-Information Systems               0.139396818 -0.7908809 1.0696745
## Management-Information Systems        0.140471818 -0.2143535 0.4952971
## Marketing-Information Systems        -0.125995829 -0.5178930 0.2659014
## Medicine-Information Systems          0.076312727 -0.5170117 0.6696372
## Psychology-Information Systems       -0.150992135 -0.5166673 0.2146830
## Transportation-Information Systems   -0.383994848 -1.1563302 0.3883405
## Management-Law                        0.001075000 -0.9337902 0.9359402
## Marketing-Law                        -0.265392647 -1.2149480 0.6841627
## Medicine-Law                         -0.063084091 -1.1119434 0.9857752
## Psychology-Law                       -0.290388953 -1.2294259 0.6486480
## Transportation-Law                   -0.523391667 -1.6829493 0.6361660
## Marketing-Management                 -0.266467647 -0.6691335 0.1361982
## Medicine-Management                  -0.064159091 -0.6646508 0.5363326
## Psychology-Management                -0.291463953 -0.6686571 0.0857292
## Transportation-Management            -0.524466667 -1.3023216 0.2533882
## Medicine-Marketing                    0.202308556 -0.4208067 0.8254239
## Psychology-Marketing                 -0.024996306 -0.4372548 0.3872622
## Transportation-Marketing             -0.257999020 -1.0534490 0.5374509
## Psychology-Medicine                  -0.227304863 -0.8342707 0.3796610
## Transportation-Medicine              -0.460307576 -1.3720044 0.4513893
## Transportation-Psychology            -0.233002713 -1.0158664 0.5498610
##                                          p adj
## Information Systems-General Interest 0.9999760
## Law-General Interest                 0.9999992
## Management-General Interest          0.9999644
## Marketing-General Interest           0.9839949
## Medicine-General Interest            1.0000000
## Psychology-General Interest          0.9636558
## Transportation-General Interest      0.8111312
## Law-Information Systems              0.9998032
## Management-Information Systems       0.9274765
## Marketing-Information Systems        0.9761743
## Medicine-Information Systems         0.9999296
## Psychology-Information Systems       0.9106921
## Transportation-Information Systems   0.7943760
## Management-Law                       1.0000000
## Marketing-Law                        0.9894312
## Medicine-Law                         0.9999996
## Psychology-Law                       0.9809031
## Transportation-Law                   0.8643371
## Marketing-Management                 0.4664602
## Medicine-Management                  0.9999801
## Psychology-Management                0.2635759
## Transportation-Management            0.4411545
## Medicine-Marketing                   0.9748210
## Psychology-Marketing                 0.9999996
## Transportation-Marketing             0.9749641
## Psychology-Medicine                  0.9454164
## Transportation-Medicine              0.7811834
## Transportation-Psychology            0.9846936
print(plist[3])
## $pubtype

print(boxplotlist[3])
## $pubtype

print(meanlist[2])
## $pubtype
## # A tibble: 3 x 2
##   `modsdata[, i]`           name
##   <fct>                    <dbl>
## 1 Conference proceedings  0.0141
## 2 Journal article        -0.279 
## 3 Working Paper          -0.416
print(aovlist[2])
## $pubtype
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   2   1.38  0.6896   1.995  0.139
## Residuals     208  71.91  0.3457
TukeyHSD(aov(cohens_d ~ pubtype, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ pubtype, data = meta_data)
## 
## $pubtype
##                                              diff        lwr        upr
## Journal article-Conference proceedings -0.2933075 -0.6779597 0.09134477
## Working Paper-Conference proceedings   -0.4304351 -0.9896559 0.12878581
## Working Paper-Journal article          -0.1371276 -0.5678069 0.29355171
##                                            p adj
## Journal article-Conference proceedings 0.1720876
## Working Paper-Conference proceedings   0.1665353
## Working Paper-Journal article          0.7329892
print(plist[4])
## $published

print(boxplotlist[4])
## $published

print(meanlist[3])
## $published
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.175
## 2 yes             -0.279
print(aovlist[3])
## $published
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.24  0.2380   0.681   0.41
## Residuals     209  73.05  0.3495
t.test(cohens_d ~ published, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by published
## t = 1.0904, df = 39.057, p-value = 0.2822
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.08884321  0.29667525
## sample estimates:
##  mean in group no mean in group yes 
##         -0.175320         -0.279236
print(plist[5])
## $within_subject

print(boxplotlist[5])
## $within_subject

print(meanlist[4])
## $within_subject
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.256
## 2 yes             -0.328
print(aovlist[4])
## $within_subject
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.14  0.1406   0.402  0.527
## Residuals     209  73.14  0.3500
t.test(cohens_d ~ within_subject, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by within_subject
## t = 0.68673, df = 46.112, p-value = 0.4957
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.1389481  0.2828644
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2560106        -0.3279687
print(plist[6])
## $field

print(boxplotlist[6])
## $field

print(meanlist[5])
## $field
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.253
## 2 yes             -0.482
print(aovlist[5])
## $field
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.64  0.6401   1.841  0.176
## Residuals     209  72.64  0.3476
t.test(cohens_d ~ field, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by field
## t = 0.88907, df = 12.601, p-value = 0.3906
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.3293329  0.7874491
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2528111        -0.4818692
print(plist[7])
## $dv_cat

print(boxplotlist[7])
## $dv_cat

print(meanlist[6])
## $dv_cat
## # A tibble: 3 x 2
##   `modsdata[, i]`         name
##   <fct>                  <dbl>
## 1 hypothetical outcome  -0.227
## 2 real outcome          -0.332
## 3 subjective perception -0.265
print(aovlist[6])
## $dv_cat
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   2   0.15  0.0758   0.216  0.806
## Residuals     208  73.13  0.3516
TukeyHSD(aov(cohens_d ~ dv_cat, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ dv_cat, data = meta_data)
## 
## $dv_cat
##                                                   diff        lwr       upr
## real outcome-hypothetical outcome          -0.10442500 -0.4824002 0.2735502
## subjective perception-hypothetical outcome -0.03768524 -0.3094730 0.2341025
## subjective perception-real outcome          0.06673976 -0.2403073 0.3737868
##                                                p adj
## real outcome-hypothetical outcome          0.7913416
## subjective perception-hypothetical outcome 0.9426722
## subjective perception-real outcome         0.8650492
print(plist[8])
## $subjective_perception

print(boxplotlist[8])
## $subjective_perception

print(meanlist[7])
## $subjective_perception
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.275
## 2 yes             -0.264
print(aovlist[7])
## $subjective_perception
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.01  0.0055   0.016    0.9
## Residuals     209  73.28  0.3506
t.test(cohens_d ~ subjective_perception, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by subjective_perception
## t = -0.11556, df = 86.378, p-value = 0.9083
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.2099490  0.1868804
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2753421        -0.2638078
print(plist[9])
## $dv_scaling

print(boxplotlist[9])
## $dv_scaling

print(meanlist[8])
## $dv_scaling
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 continuous      -0.293
## 2 discrete         0.127
print(aovlist[8])
## $dv_scaling
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   2.14  2.1447   6.301 0.0128 *
## Residuals     209  71.14  0.3404                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ dv_scaling, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by dv_scaling
## t = -1.6169, df = 12.577, p-value = 0.1307
## alternative hypothesis: true difference in means between group continuous and group discrete is not equal to 0
## 95 percent confidence interval:
##  -0.9814309  0.1428399
## sample estimates:
## mean in group continuous   mean in group discrete 
##               -0.2927571                0.1265385
print(plist[10])
## $dv_measurement_type

print(boxplotlist[10])
## $dv_measurement_type

print(meanlist[9])
## $dv_measurement_type
## # A tibble: 2 x 2
##   `modsdata[, i]`    name
##   <fct>             <dbl>
## 1 behavioral      -0.0456
## 2 self-report     -0.314
print(aovlist[9])
## $dv_measurement_type
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   2.20  2.1988   6.465 0.0117 *
## Residuals     209  71.09  0.3401                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ dv_measurement_type, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by dv_measurement_type
## t = 2.0777, df = 44.495, p-value = 0.04354
## alternative hypothesis: true difference in means between group behavioral and group self-report is not equal to 0
## 95 percent confidence interval:
##  0.008135755 0.528755796
## sample estimates:
##  mean in group behavioral mean in group self-report 
##               -0.04555135               -0.31399713
t.test(cohens_d ~ dv_measurement_type, data = meta_data, var.equal = TRUE)
## 
##  Two Sample t-test
## 
## data:  cohens_d by dv_measurement_type
## t = 2.5426, df = 209, p-value = 0.01173
## alternative hypothesis: true difference in means between group behavioral and group self-report is not equal to 0
## 95 percent confidence interval:
##  0.06030649 0.47658506
## sample estimates:
##  mean in group behavioral mean in group self-report 
##               -0.04555135               -0.31399713
print(plist[11])
## $compensation

print(boxplotlist[11])
## $compensation

print(meanlist[10])
## $compensation
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.300
## 2 yes             -0.254
print(aovlist[10])
## $compensation
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.09  0.0888   0.253  0.615
## Residuals     209  73.20  0.3502
t.test(cohens_d ~ compensation, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by compensation
## t = -0.49117, df = 100.62, p-value = 0.6244
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.2302643  0.1388717
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2998424        -0.2541461
print(plist[12])
## $incentive_compatible

print(boxplotlist[12])
## $incentive_compatible

print(meanlist[11])
## $incentive_compatible
## # A tibble: 2 x 2
##   `modsdata[, i]`    name
##   <fct>             <dbl>
## 1 no              -0.289 
## 2 yes             -0.0200
print(aovlist[11])
## $incentive_compatible
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.13  1.1274   3.266 0.0722 .
## Residuals     209  72.16  0.3452                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ incentive_compatible, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by incentive_compatible
## t = -1.7025, df = 18.518, p-value = 0.1054
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.5993298  0.0621815
## sample estimates:
##  mean in group no mean in group yes 
##       -0.28856237       -0.01998824
print(plist[13])
## $preregistered

print(boxplotlist[13])
## $preregistered

print(meanlist[12])
## $preregistered
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.277
## 2 yes             -0.226
print(aovlist[12])
## $preregistered
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.08  0.0836   0.239  0.626
## Residuals     209  73.20  0.3502
t.test(cohens_d ~ preregistered, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by preregistered
## t = -0.46205, df = 55.277, p-value = 0.6459
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.2709366  0.1694007
## sample estimates:
##  mean in group no mean in group yes 
##         -0.276548         -0.225780
print(plist[14])
## $online

print(boxplotlist[14])
## $online

print(meanlist[13])
## $online
## # A tibble: 3 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.254
## 2 yes             -0.298
## 3 <NA>             0.102
print(aovlist[13])
## $online
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.10  0.0976   0.281  0.597
## Residuals     204  70.87  0.3474               
## 5 observations deleted due to missingness
t.test(cohens_d ~ online, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by online
## t = 0.52919, df = 197.77, p-value = 0.5973
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.1186933  0.2057591
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2543279        -0.2978608
print(plist[15])
## $students

print(boxplotlist[15])
## $students

print(meanlist[14])
## $students
## # A tibble: 3 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.261
## 2 yes             -0.335
## 3 <NA>             0.102
print(aovlist[14])
## $students
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.19  0.1852   0.534  0.466
## Residuals     204  70.78  0.3470               
## 5 observations deleted due to missingness
t.test(cohens_d ~ students, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by students
## t = 0.83269, df = 77.193, p-value = 0.4076
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.1035481  0.2524025
## sample estimates:
##  mean in group no mean in group yes 
##        -0.2607085        -0.3351357
print(plist[16])
## $location

print(boxplotlist[16])
## $location

print(meanlist[15])
## $location
## # A tibble: 12 x 2
##    `modsdata[, i]`    name
##    <fct>             <dbl>
##  1 America         -0.388 
##  2 Asia            -0.731 
##  3 Canada          -0.671 
##  4 China           -0.631 
##  5 Germany         -0.150 
##  6 Italy           -0.685 
##  7 multiple        -0.475 
##  8 Netherlands     -0.0846
##  9 Non-Us          -0.071 
## 10 Turkey          -0.596 
## 11 United States   -0.184 
## 12 <NA>            -0.171
print(aovlist[15])
## $location
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]  10   6.08  0.6077   1.637  0.103
## Residuals     131  48.65  0.3714               
## 69 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ location, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ location, data = meta_data)
## 
## $location
##                                  diff        lwr       upr     p adj
## Asia-America              -0.34322273 -1.5090052 0.8225597 0.9966006
## Canada-America            -0.28303939 -1.1164810 0.5504022 0.9894731
## China-America             -0.24258939 -1.2559190 0.7707402 0.9994125
## Germany-America            0.23762727 -1.2971967 1.7724512 0.9999892
## Italy-America             -0.29667273 -1.8314967 1.2381512 0.9999140
## multiple-America          -0.08692828 -0.8510537 0.6771971 0.9999994
## Netherlands-America        0.30349394 -0.5939258 1.2009137 0.9898045
## Non-Us-America             0.31712727 -0.5552644 1.1895190 0.9824813
## Turkey-America            -0.20753939 -1.5080242 1.0929454 0.9999856
## United States-America      0.20363958 -0.4473170 0.8545961 0.9944528
## Canada-Asia                0.06018333 -1.0925732 1.2129399 1.0000000
## China-Asia                 0.10063333 -1.1881877 1.3894543 1.0000000
## Germany-Asia               0.58085000 -1.1482848 2.3099848 0.9903166
## Italy-Asia                 0.04655000 -1.6825848 1.7756848 1.0000000
## multiple-Asia              0.25629444 -0.8473859 1.3599748 0.9995506
## Netherlands-Asia           0.64671667 -0.5531104 1.8465437 0.7975694
## Non-Us-Asia                0.66035000 -0.5208740 1.8415740 0.7591473
## Turkey-Asia                0.13568333 -1.3892702 1.6606369 0.9999999
## United States-Asia         0.54686231 -0.4817130 1.5754376 0.8108605
## China-Canada               0.04045000 -0.9578665 1.0387665 1.0000000
## Germany-Canada             0.52066667 -1.0042869 2.0456202 0.9890333
## Italy-Canada              -0.01363333 -1.5385869 1.5113202 1.0000000
## multiple-Canada            0.19611111 -0.5479900 0.9402123 0.9986517
## Netherlands-Canada         0.58653333 -0.2938990 1.4669657 0.5211897
## Non-Us-Canada              0.60016667 -0.2547405 1.4550738 0.4401527
## Turkey-Canada              0.07550000 -1.2133210 1.3643210 1.0000000
## United States-Canada       0.48667897 -0.1406514 1.1140093 0.2907075
## Germany-China              0.48021667 -1.1500273 2.1104606 0.9965861
## Italy-China               -0.05408333 -1.6843273 1.5761606 1.0000000
## multiple-China             0.15566111 -0.7855607 1.0968829 0.9999798
## Netherlands-China          0.54608333 -0.5062346 1.5984013 0.8328428
## Non-Us-China               0.55971667 -0.4713401 1.5907735 0.7904237
## Turkey-China               0.03505000 -1.3767827 1.4468827 1.0000000
## United States-China        0.44622897 -0.4056837 1.2981417 0.8246160
## Italy-Germany             -0.53430000 -2.5309329 1.4623329 0.9984654
## multiple-Germany          -0.32455556 -1.8127579 1.1636467 0.9997430
## Netherlands-Germany        0.06586667 -1.4949731 1.6267064 1.0000000
## Non-Us-Germany             0.07950000 -1.4670852 1.6260852 1.0000000
## Turkey-Germany            -0.44516667 -2.2678348 1.3775015 0.9993002
## United States-Germany     -0.03398769 -1.4673763 1.3994009 1.0000000
## multiple-Italy             0.20974444 -1.2784579 1.6979467 0.9999956
## Netherlands-Italy          0.60016667 -0.9606731 2.1610064 0.9737582
## Non-Us-Italy               0.61380000 -0.9327852 2.1603852 0.9672811
## Turkey-Italy               0.08913333 -1.7335348 1.9118015 1.0000000
## United States-Italy        0.50031231 -0.9330763 1.9337009 0.9870432
## Netherlands-multiple       0.39042222 -0.4246997 1.2055442 0.8921766
## Non-Us-multiple            0.40405556 -0.3834271 1.1915382 0.8424374
## Turkey-multiple           -0.12061111 -1.3657305 1.1245083 0.9999999
## United States-multiple     0.29056786 -0.2412272 0.8223630 0.7837615
## Non-Us-Netherlands         0.01363333 -0.9037562 0.9310228 1.0000000
## Turkey-Netherlands        -0.51103333 -1.8421219 0.8200553 0.9740430
## United States-Netherlands -0.09985436 -0.8099816 0.6102728 0.9999957
## Turkey-Non-Us             -0.52466667 -1.8390114 0.7896780 0.9659491
## United States-Non-Us      -0.11348769 -0.7917102 0.5647348 0.9999775
## United States-Turkey       0.41117897 -0.7678796 1.5902375 0.9871285
print(plist[17])
## $US

print(boxplotlist[17])
## $US

print(meanlist[16])
## $US
## # A tibble: 3 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.410
## 2 yes             -0.214
## 3 <NA>            -0.234
print(aovlist[16])
## $US
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.13  1.1337   3.222 0.0751 .
## Residuals     122  42.92  0.3518                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 87 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ US, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ US, data = meta_data)
## 
## $US
##            diff         lwr       upr     p adj
## yes-no 0.196309 -0.02017535 0.4127933 0.0751119
t.test(cohens_d ~ US, data = meta_data, var.equal = TRUE, na.rm = TRUE)
## 
##  Two Sample t-test
## 
## data:  cohens_d by US
## t = -1.7951, df = 122, p-value = 0.07511
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.41279333  0.02017535
## sample estimates:
##  mean in group no mean in group yes 
##        -0.4102708        -0.2139618
print(plist[18])
## $sample_type

print(boxplotlist[18])
## $sample_type

print(meanlist[17])
## $sample_type
## # A tibble: 4 x 2
##   `modsdata[, i]`                   name
##   <fct>                            <dbl>
## 1 mturkers/ prolific participants -0.298
## 2 others                          -0.200
## 3 students                        -0.335
## 4 <NA>                             0.102
print(aovlist[17])
## $sample_type
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   2   0.56  0.2788   0.804  0.449
## Residuals     203  70.41  0.3469               
## 5 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ sample_type, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ sample_type, data = meta_data)
## 
## $sample_type
##                                                 diff        lwr       upr
## others-mturkers/ prolific participants    0.09827369 -0.1256503 0.3221977
## students-mturkers/ prolific participants -0.03727493 -0.2922109 0.2176611
## students-others                          -0.13554862 -0.4134373 0.1423400
##                                              p adj
## others-mturkers/ prolific participants   0.5549304
## students-mturkers/ prolific participants 0.9364443
## students-others                          0.4835412
print(plist[19])
## $domain

print(boxplotlist[19])
## $domain

print(meanlist[18])
## $domain
## # A tibble: 15 x 2
##    `modsdata[, i]`       name
##    <fct>                <dbl>
##  1 airport security   -0.0873
##  2 art                -0.314 
##  3 autonomous driving -0.769 
##  4 baking/cooking     -0.372 
##  5 finance            -0.518 
##  6 law                -0.173 
##  7 management         -0.203 
##  8 medicine           -0.397 
##  9 military           -1.26  
## 10 multiple           -0.268 
## 11 news                0.0334
## 12 other               0.0882
## 13 real estate        -0.685 
## 14 shopping            0.102 
## 15 social              0.0136
print(aovlist[18])
## $domain
##                Df Sum Sq Mean Sq F value  Pr(>F)   
## modsdata[, i]  14  10.08  0.7202   2.233 0.00792 **
## Residuals     196  63.20  0.3225                   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
TukeyHSD(aov(cohens_d ~ domain, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ domain, data = meta_data)
## 
## $domain
##                                            diff         lwr        upr
## art-airport security                -0.22666667 -1.81943132 1.36609799
## autonomous driving-airport security -0.68185238 -2.02798421 0.66427944
## baking/cooking-airport security     -0.28450000 -1.87726465 1.30826465
## finance-airport security            -0.43020000 -1.73068689 0.87028689
## law-airport security                -0.08605897 -1.33552637 1.16340842
## management-airport security         -0.11568229 -1.26803128 1.03666670
## medicine-airport security           -0.30962276 -1.47635453 0.85710900
## military-airport security           -1.17733333 -2.77009799 0.41543132
## multiple-airport security           -0.18082843 -1.35572062 0.99406375
## news-airport security                0.12070000 -1.47206465 1.71346465
## other-airport security               0.17558333 -1.14506735 1.49623402
## real estate-airport security        -0.59746667 -2.37823169 1.18329835
## shopping-airport security            0.18905556 -1.11143134 1.48954245
## social-airport security              0.10094444 -1.19954245 1.40143134
## autonomous driving-art              -0.45518571 -1.80131754 0.89094611
## baking/cooking-art                  -0.05783333 -1.65059799 1.53493132
## finance-art                         -0.20353333 -1.50402023 1.09695356
## law-art                              0.14060769 -1.10885970 1.39007509
## management-art                       0.11098437 -1.04136462 1.26333337
## medicine-art                        -0.08295610 -1.24968786 1.08377567
## military-art                        -0.95066667 -2.54343132 0.64209799
## multiple-art                         0.04583824 -1.12905395 1.22073042
## news-art                             0.34736667 -1.24539799 1.94013132
## other-art                            0.40225000 -0.91840068 1.72290068
## real estate-art                     -0.37080000 -2.15156502 1.40996502
## shopping-art                         0.41572222 -0.88476467 1.71620912
## social-art                           0.32761111 -0.97287578 1.62809801
## baking/cooking-autonomous driving    0.39735238 -0.94877944 1.74348421
## finance-autonomous driving           0.25165238 -0.73142331 1.23472807
## law-autonomous driving               0.59579341 -0.31872308 1.51030990
## management-autonomous driving        0.56617009 -0.21041207 1.34275224
## medicine-autonomous driving          0.37222962 -0.42553893 1.16999816
## military-autonomous driving         -0.49548095 -1.84161278 0.85065087
## multiple-autonomous driving          0.50102395 -0.30863233 1.31068023
## news-autonomous driving              0.80255238 -0.54357944 2.14868421
## other-autonomous driving             0.85743571 -0.15216315 1.86703458
## real estate-autonomous driving       0.08438571 -1.47967813 1.64844956
## shopping-autonomous driving          0.87090794 -0.11216775 1.85398362
## social-autonomous driving            0.78279683 -0.20027886 1.76587251
## finance-baking/cooking              -0.14570000 -1.44618689 1.15478689
## law-baking/cooking                   0.19844103 -1.05102637 1.44790842
## management-baking/cooking            0.16881771 -0.98353128 1.32116670
## medicine-baking/cooking             -0.02512276 -1.19185453 1.14160900
## military-baking/cooking             -0.89283333 -2.48559799 0.69993132
## multiple-baking/cooking              0.10367157 -1.07122062 1.27856375
## news-baking/cooking                  0.40520000 -1.18756465 1.99796465
## other-baking/cooking                 0.46008333 -0.86056735 1.78073402
## real estate-baking/cooking          -0.31296667 -2.09373169 1.46779835
## shopping-baking/cooking              0.47355556 -0.82693134 1.77404245
## social-baking/cooking                0.38544444 -0.91504245 1.68593134
## law-finance                          0.34414103 -0.50175240 1.19003445
## management-finance                   0.31451771 -0.37994260 1.00897801
## medicine-finance                     0.12057724 -0.59749653 0.83865100
## military-finance                    -0.74713333 -2.04762023 0.55335356
## multiple-finance                     0.24937157 -0.48188664 0.98062978
## news-finance                         0.55090000 -0.74958689 1.85138689
## other-finance                        0.60578333 -0.34210123 1.55366790
## real estate-finance                 -0.16726667 -1.69222272 1.35768939
## shopping-finance                     0.61925556 -0.30032755 1.53883866
## social-finance                       0.53114444 -0.38843866 1.45072755
## management-law                      -0.02962332 -0.62306895 0.56382232
## medicine-law                        -0.22356379 -0.84447645 0.39734887
## military-law                        -1.09127436 -2.34074175 0.15819303
## multiple-law                        -0.09476946 -0.73088357 0.54134466
## news-law                             0.20675897 -1.04270842 1.45622637
## other-law                            0.26164231 -0.61493499 1.13821960
## real estate-law                     -0.51140769 -1.99309375 0.97027837
## shopping-law                         0.27511453 -0.57077889 1.12100795
## social-law                           0.18700342 -0.65889001 1.03289684
## medicine-management                 -0.19394047 -0.58416088 0.19627993
## military-management                 -1.06165104 -2.21400003 0.09069795
## multiple-management                 -0.06514614 -0.47912754 0.34883526
## news-management                      0.23638229 -0.91596670 1.38873128
## other-management                     0.29126563 -0.44025825 1.02278950
## real estate-management              -0.48178437 -1.88254596 0.91897721
## shopping-management                  0.30473785 -0.38972246 0.99919815
## social-management                    0.21662674 -0.47783357 0.91108704
## military-medicine                   -0.86771057 -2.03444233 0.29902120
## multiple-medicine                    0.12879433 -0.32368264 0.58127131
## news-medicine                        0.43032276 -0.73640900 1.59705453
## other-medicine                       0.48520610 -0.26877135 1.23918355
## real estate-medicine                -0.28784390 -1.70046127 1.12477346
## shopping-medicine                    0.49867832 -0.21939544 1.21675208
## social-medicine                      0.41056721 -0.30750655 1.12864097
## multiple-military                    0.99650490 -0.17838728 2.17139709
## news-military                        1.29803333 -0.29473132 2.89079799
## other-military                       1.35291667  0.03226598 2.67356735
## real estate-military                 0.57986667 -1.20089835 2.36063169
## shopping-military                    1.36638889  0.06590199 2.66687578
## social-military                      1.27827778 -0.02220912 2.57876467
## news-multiple                        0.30152843 -0.87336375 1.47642062
## other-multiple                       0.35641176 -0.41013284 1.12295637
## real estate-multiple                -0.41663824 -1.83600304 1.00272657
## shopping-multiple                    0.36988399 -0.36137422 1.10114219
## social-multiple                      0.28177288 -0.44948533 1.01303108
## other-news                           0.05488333 -1.26576735 1.37553402
## real estate-news                    -0.71816667 -2.49893169 1.06259835
## shopping-news                        0.06835556 -1.23213134 1.36884245
## social-news                         -0.01975556 -1.32024245 1.28073134
## real estate-other                   -0.77305000 -2.31523775 0.76913775
## shopping-other                       0.01347222 -0.93441234 0.96135679
## social-other                        -0.07463889 -1.02252345 0.87324568
## shopping-real estate                 0.78652222 -0.73843383 2.31147828
## social-real estate                   0.69841111 -0.82654495 2.22336717
## social-shopping                     -0.08811111 -1.00769421 0.83147199
##                                         p adj
## art-airport security                0.9999999
## autonomous driving-airport security 0.9178298
## baking/cooking-airport security     0.9999990
## finance-airport security            0.9983314
## law-airport security                1.0000000
## management-airport security         1.0000000
## medicine-airport security           0.9998612
## military-airport security           0.4178332
## multiple-airport security           0.9999999
## news-airport security               1.0000000
## other-airport security              1.0000000
## real estate-airport security        0.9980597
## shopping-airport security           0.9999999
## social-airport security             1.0000000
## autonomous driving-art              0.9978922
## baking/cooking-art                  1.0000000
## finance-art                         0.9999998
## law-art                             1.0000000
## management-art                      1.0000000
## medicine-art                        1.0000000
## military-art                        0.7653385
## multiple-art                        1.0000000
## news-art                            0.9999872
## other-art                           0.9993237
## real estate-art                     0.9999929
## shopping-art                        0.9988473
## social-art                          0.9999253
## baking/cooking-autonomous driving   0.9995247
## finance-autonomous driving          0.9999095
## law-autonomous driving              0.6371411
## management-autonomous driving       0.4422395
## medicine-autonomous driving         0.9562787
## military-autonomous driving         0.9949489
## multiple-autonomous driving         0.7160996
## news-autonomous driving             0.7667898
## other-autonomous driving            0.1977467
## real estate-autonomous driving      1.0000000
## shopping-autonomous driving         0.1466386
## social-autonomous driving           0.2918953
## finance-baking/cooking              1.0000000
## law-baking/cooking                  0.9999998
## management-baking/cooking           0.9999999
## medicine-baking/cooking             1.0000000
## military-baking/cooking             0.8373303
## multiple-baking/cooking             1.0000000
## news-baking/cooking                 0.9999160
## other-baking/cooking                0.9971190
## real estate-baking/cooking          0.9999992
## shopping-baking/cooking             0.9954671
## social-baking/cooking               0.9995023
## law-finance                         0.9867181
## management-finance                  0.9657740
## medicine-finance                    0.9999996
## military-finance                    0.8112385
## multiple-finance                    0.9976954
## news-finance                        0.9808083
## other-finance                       0.6677125
## real estate-finance                 1.0000000
## shopping-finance                    0.5819427
## social-finance                      0.8051843
## management-law                      1.0000000
## medicine-law                        0.9959563
## military-law                        0.1628444
## multiple-law                        0.9999999
## news-law                            0.9999996
## other-law                           0.9994611
## real estate-law                     0.9973850
## shopping-law                        0.9986096
## social-law                          0.9999849
## medicine-management                 0.9285569
## military-management                 0.1074304
## multiple-management                 0.9999998
## news-management                     0.9999941
## other-management                    0.9891444
## real estate-management              0.9974791
## shopping-management                 0.9738332
## social-management                   0.9991188
## military-medicine                   0.4070665
## multiple-medicine                   0.9996860
## news-medicine                       0.9948449
## other-medicine                      0.6568358
## real estate-medicine                0.9999946
## shopping-medicine                   0.5284745
## social-medicine                     0.8165274
## multiple-military                   0.1994825
## news-military                       0.2560024
## other-military                      0.0386828
## real estate-military                0.9985914
## shopping-military                   0.0290837
## social-military                     0.0595101
## news-multiple                       0.9999067
## other-multiple                      0.9574999
## real estate-multiple                0.9995538
## shopping-multiple                   0.9186630
## social-multiple                     0.9920665
## other-news                          1.0000000
## real estate-news                    0.9877576
## shopping-news                       1.0000000
## social-news                         1.0000000
## real estate-other                   0.9238779
## shopping-other                      1.0000000
## social-other                        1.0000000
## shopping-real estate                0.9064423
## social-real estate                  0.9624429
## social-shopping                     1.0000000
print(plist[20])
## $management

print(boxplotlist[20])
## $management

print(meanlist[19])
## $management
## # A tibble: 3 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.319
## 2 yes             -0.179
## 3 <NA>            -0.229
print(aovlist[19])
## $management
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.93  0.9311   2.637  0.106
## Residuals     203  71.68  0.3531               
## 6 observations deleted due to missingness
t.test(cohens_d ~ management, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by management
## t = -1.4721, df = 114.53, p-value = 0.1437
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.3282030  0.0483646
## sample estimates:
##  mean in group no mean in group yes 
##        -0.3192138        -0.1792947
print(plist[21])
## $human_type

print(boxplotlist[21])
## $human_type

print(meanlist[20])
## $human_type
## # A tibble: 7 x 2
##   `modsdata[, i]`             name
##   <fct>                      <dbl>
## 1 a lay person/ lay people -0.257 
## 2 expert(s)                -0.350 
## 3 human-controlled machine -0.268 
## 4 multiple                  0.0402
## 5 other                    -0.150 
## 6 other participant(s)      0.169 
## 7 participants themselves   0.733
print(aovlist[20])
## $human_type
##                Df Sum Sq Mean Sq F value   Pr(>F)    
## modsdata[, i]   6   8.83  1.4720   4.659 0.000183 ***
## Residuals     204  64.45  0.3159                     
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
TukeyHSD(aov(cohens_d ~ human_type, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ human_type, data = meta_data)
## 
## $human_type
##                                                          diff         lwr
## expert(s)-a lay person/ lay people                -0.09299288 -0.51986888
## human-controlled machine-a lay person/ lay people -0.01089412 -0.86248502
## multiple-a lay person/ lay people                  0.29700588 -0.55458502
## other-a lay person/ lay people                     0.10630588 -1.14500862
## other participant(s)-a lay person/ lay people      0.42558824 -0.14855424
## participants themselves-a lay person/ lay people   0.98970588  0.05948876
## human-controlled machine-expert(s)                 0.08209876 -0.67802663
## multiple-expert(s)                                 0.38999876 -0.37012663
## other-expert(s)                                    0.19929876 -0.99165529
## other participant(s)-expert(s)                     0.51858111  0.09170511
## participants themselves-expert(s)                  1.08269876  0.23541638
## multiple-human-controlled machine                  0.30790000 -0.75076641
## other-human-controlled machine                     0.11720000 -1.28328402
## other participant(s)-human-controlled machine      0.43648235 -0.41510855
## participants themselves-human-controlled machine   1.00060000 -0.12228529
## other-multiple                                    -0.19070000 -1.59118402
## other participant(s)-multiple                      0.12858235 -0.72300855
## participants themselves-multiple                   0.69270000 -0.43018529
## other participant(s)-other                         0.31928235 -0.93203215
## participants themselves-other                      0.88340000 -0.56623868
## participants themselves-other participant(s)       0.56411765 -0.36609947
##                                                         upr     p adj
## expert(s)-a lay person/ lay people                0.3338831 0.9950206
## human-controlled machine-a lay person/ lay people 0.8406968 1.0000000
## multiple-a lay person/ lay people                 1.1485968 0.9444070
## other-a lay person/ lay people                    1.3576204 0.9999779
## other participant(s)-a lay person/ lay people     0.9997307 0.2960924
## participants themselves-a lay person/ lay people  1.9199230 0.0288586
## human-controlled machine-expert(s)                0.8422242 0.9999091
## multiple-expert(s)                                1.1501242 0.7277213
## other-expert(s)                                   1.3902528 0.9988564
## other participant(s)-expert(s)                    0.9454571 0.0067778
## participants themselves-expert(s)                 1.9299811 0.0034893
## multiple-human-controlled machine                 1.3665664 0.9771064
## other-human-controlled machine                    1.5176840 0.9999798
## other participant(s)-human-controlled machine     1.2880733 0.7286717
## participants themselves-human-controlled machine  2.1234853 0.1157729
## other-multiple                                    1.2097840 0.9996494
## other participant(s)-multiple                     0.9801733 0.9993641
## participants themselves-multiple                  1.8155853 0.5246372
## other participant(s)-other                        1.5705969 0.9883629
## participants themselves-other                     2.3330387 0.5396541
## participants themselves-other participant(s)      1.4943348 0.5455840
print(plist[22])
## $expert

print(boxplotlist[22])
## $expert

print(meanlist[21])
## $expert
## # A tibble: 3 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.103
## 2 yes             -0.346
## 3 <NA>             0.348
print(aovlist[21])
## $expert
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.99  1.9914   6.237 0.0133 *
## Residuals     200  63.86  0.3193                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 9 observations deleted due to missingness
t.test(cohens_d ~ expert, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by expert
## t = 2.8976, df = 84.418, p-value = 0.004788
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  0.07610675 0.40901488
## sample estimates:
##  mean in group no mean in group yes 
##        -0.1033977        -0.3459585
t.test(cohens_d ~ expert, data = meta_data, var.equal = TRUE)
## 
##  Two Sample t-test
## 
## data:  cohens_d by expert
## t = 2.4973, df = 200, p-value = 0.01332
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  0.05103529 0.43408635
## sample estimates:
##  mean in group no mean in group yes 
##        -0.1033977        -0.3459585
print(plist[23])
## $algorithm_dummy

print(boxplotlist[23])
## $algorithm_dummy

print(meanlist[22])
## $algorithm_dummy
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.339
## 2 yes             -0.191
print(aovlist[22])
## $algorithm_dummy
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.16  1.1556   3.349 0.0687 .
## Residuals     209  72.13  0.3451                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ algorithm_dummy, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by algorithm_dummy
## t = -1.8311, df = 208.92, p-value = 0.06851
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.30744914  0.01134198
## sample estimates:
##  mean in group no mean in group yes 
##        -0.3391963        -0.1911427
print(plist[24])
## $ai_dummy

print(boxplotlist[24])
## $ai_dummy

print(meanlist[23])
## $ai_dummy
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.321
## 2 yes             -0.101
print(aovlist[23])
## $ai_dummy
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.89  1.8923    5.54 0.0195 *
## Residuals     209  71.39  0.3416                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ ai_dummy, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by ai_dummy
## t = -2.4515, df = 93.233, p-value = 0.01609
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.39775444 -0.04174887
## sample estimates:
##  mean in group no mean in group yes 
##        -0.3210805        -0.1013288
t.test(cohens_d ~ ai_dummy, data = meta_data, var.equal = TRUE)
## 
##  Two Sample t-test
## 
## data:  cohens_d by ai_dummy
## t = -2.3536, df = 209, p-value = 0.01952
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.40381343 -0.03568988
## sample estimates:
##  mean in group no mean in group yes 
##        -0.3210805        -0.1013288
print(plist[25])
## $textual

print(boxplotlist[25])
## $textual

print(meanlist[24])
## $textual
## # A tibble: 2 x 2
##   `modsdata[, i]`   name
##   <fct>            <dbl>
## 1 no              -0.182
## 2 yes             -0.283
print(aovlist[24])
## $textual
##                Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i]   1   0.29  0.2912   0.834  0.362
## Residuals     209  72.99  0.3493
t.test(cohens_d ~ textual, data = meta_data, var.equal = FALSE)
## 
##  Welch Two Sample t-test
## 
## data:  cohens_d by textual
## t = 0.81645, df = 42.59, p-value = 0.4188
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
##  -0.1486173  0.3507121
## sample estimates:
##  mean in group no mean in group yes 
##        -0.1821588        -0.2832062
print(plist[26])
## $severity

print(boxplotlist[26])
## $severity

print(meanlist[25])
## $severity
## # A tibble: 4 x 2
##   `modsdata[, i]`    name
##   <fct>             <dbl>
## 1 low             -0.0295
## 2 medium          -0.278 
## 3 high            -0.478 
## 4 <NA>            -0.131
print(aovlist[25])
## $severity
##                Df Sum Sq Mean Sq F value  Pr(>F)   
## modsdata[, i]   2   4.75  2.3727   6.543 0.00182 **
## Residuals     173  62.74  0.3626                   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 35 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ severity, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ severity, data = meta_data)
## 
## $severity
##                   diff        lwr         upr     p adj
## medium-low  -0.2489130 -0.5311459  0.03331997 0.0959302
## high-low    -0.4486311 -0.7428408 -0.15442154 0.0011826
## high-medium -0.1997182 -0.4437381  0.04430174 0.1320253
print(plist[27])
## $subj_vs_obj

print(boxplotlist[27])
## $subj_vs_obj

print(meanlist[26])
## $subj_vs_obj
## # A tibble: 3 x 2
##   `modsdata[, i]`    name
##   <fct>             <dbl>
## 1 objective       -0.299 
## 2 subjective      -0.0841
## 3 <NA>            -0.324
print(aovlist[26])
## $subj_vs_obj
##                Df Sum Sq Mean Sq F value Pr(>F)  
## modsdata[, i]   1   1.28  1.2791    3.61  0.059 .
## Residuals     185  65.54  0.3543                 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 24 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ subj_vs_obj, data = meta_data))
##   Tukey multiple comparisons of means
##     95% family-wise confidence level
## 
## Fit: aov(formula = cohens_d ~ subj_vs_obj, data = meta_data)
## 
## $subj_vs_obj
##                           diff         lwr       upr     p adj
## subjective-objective 0.2144294 -0.00820868 0.4370675 0.0589714

Meta-Analytic Modeling

General Notes:

  • Goal: calculate a summary (combined) intervention effect estimate.
  • The overall effect is calculated as a weighted average of the intervention effects estimated in the individual studies.
  • The bigger the weight given to a study, the more it will contribute to the weighted average. The weight that is used is usually a value reflecting the sampling accuracy of the effect size. If an effect size has good sampling accuracy (i.e., it is likely to be an accurate reflection of reality) then it is weighted highly, whereas effect sizes that are imprecise estimates are given less weight in the calculations. Typically, the weight used is the sample size or some function of it.
  • The standard error of the summary intervention effect can be used to derive a confidence interval, which communicates the precision (or uncertainty) of the summary estimate; and to derive a P value, which communicates the strength of the evidence against the null hypothesis of no intervention effect.
  • There are two ways to conceptualise meta-analysis: fixed effects and random effects models.
  • The fixed-effect model assumes that studies in the meta-analysis are sampled from a population in which the average effect size is fixed. Put another way, sample effect sizes should be homogeneous because they come from the same population with a fixed average effect.
  • The alternative assumption is that the avrage effect size in the population varies randomly from study to study: studies in a meta-analysis come from populations that have different average effect sizes, so, population effect sizes can be thought of as being samples from a superpopulation. Put another way, the effect sizes should be heterogeneous because they come from populations with varying average effect sizes.
  • Stats experts point out that there are rarely grounds to assume the fixed-effects case, so random effect models are preferred.
  • In addition to yielding a summary quantification of the intervention effect, all methods of meta-analysis incorporate an assessment of whether the variation among the results of the separate studies is compatible with random variation, or whether it is large enough to indicate inconsistency of intervention effects across studies (heterogeneity). A formal test of this is the chi-squared test. It assesses whether observed differences in results are compatible with chance alone. A statistically significant result (P value below threshold or a large Chi2 statistic relative to its degree of freedom) provides evidence of heterogeneity.
  • Despite the mere testing of whether heterogeneity is present, methods have been developed to estimate its impact on the meta-analysis: I2.
  • I2 describes the percentage of the variability in effect estimates that is due to heterogeneity rather than sampling error (chance).
  • Thresholds for the interpretation of the I2 statistic:
    • 0% to 40%: might not be important
    • 30% to 60%: may represent moderate heterogeneity
    • 50% to 90%: may represent substantial heterogeneity
    • 75% to 100%: considerable heterogeneity
  • Types of Models: Random Effects Model, Mixed Effects Model, Multi-Level Random Effects Model

Funnel Plot (Raw Data)

General Notes:

  • Method to visually inspect the existence of a publication bias.
  • Publication bias = selective reporting of significant results that distort meta-analytic estimates (Buecker et al. 2021). Often mentioned as the main criticism against meta-analysis BUT MA allows to identify and/or accommodate publication bias (Tammo Bijmolt Slides with sources)
  • A funnel plot is a simple scatter plot of the intervention effect estimates from individual studies against some measure of each study’s size or precision.
  • Plot of sample size, precision (1/s.e.) or s.e. versus effect size
  • Reasoning: If the true effect size is non-zero, studies with small sample sizes (large s.e.) may still find non-significant, close-to-zero effects, which might not be published.
  • No publication bias: Mean effect size is the same for all sample sizes. Plot is symmetric, no empty spaces
  • Publication bias: Funnel plot is asymmetric
  • If a funnel plot is asymmetric: can indicate publication bias. But this can also be due to other causes (moderators), small studies may have lower effect sizes because of other research methods, etc.
  • Option: Funnel plot based on the error term (instead of the effect size) after meta-regression.

Interpretations:

  • The funnel plot is slightly asymmetric, with a majority of the smaller studies clustering to the right of the mean. This might be evidence of bias but also a starting point for the assessment of moderators and a focus on meta-regression.
funnel(as.numeric(meta_data$cohens_d), vi=as.numeric(meta_data$var_d), yaxis="sei", data = meta_data,
       main = "Funnel Plot Raw Data", label = 5, offset=1, back = "lightgrey", pch = 20)

Multi-Level Random-Effects Meta-Regression Model

General Notes:

  • Two steps in the HLM meta-regression:
      1. Variance components model, no explanatory variable at any level (moderators); only an intercept
      1. Model(-s) with explanatory variables (moderators)

1. Variance Components Model (Model Without Moderators)

Model Output

General Notes:

  • Model output includes
    • Overall weighted average effect size
    • Variance components per level
      • Variance between studies (level-2 units)
      • Variance within studies (level-1 units)
      • From these we can compute intra-class-correlation:
        • Proportion of the total variance that is between studies
        • Formula: Between-study Variation / Total Variation
        • If result is non-zero or even close to 1: larger variance at the higher level; which makes Multi-Level modeling more essential compared to regular OLS.

Interpretations:

  • We have an intra-class-correlation of 0.24 –> non-zero –> multi-level modelling is essential
# Estimates Multi-level random-effects meta-regression model, without moderators
multilevel <- rma.mv(cohens_d, var_d, random = ~ 1 |
study_id/es_id, data = meta_data, method="ML")
multilevel
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0782  0.2797     99     no        study_id 
## sigma^2.2  0.2523  0.5023    211     no  study_id/es_id 
## 
## Test for Heterogeneity:
## Q(df = 210) = 20928.2790, p-val < .0001
## 
## Model Results:
## 
## estimate      se     zval    pval    ci.lb    ci.ub 
##  -0.2464  0.0483  -5.1016  <.0001  -0.3411  -0.1517  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
# intra-class correlation
(0.0782)/(0.0782+0.2523)
## [1] 0.2366112

Outlier Detection

  • Standardized residuals (observed residuals divided by the corresponding standard errors)
plot(standresid1, type="o", pch=19, xlab="Observed effect size", ylab="Standardized residual")

  • Cook’s distance D for each observed effect size (change in beta if one observation is dropped)
    • Thresholds: D > 0.5 and > 1.0; D > 3*mean; D > 4/n; D > chisq(# regrcoeff, 0.50); any extreme D-value
  • Studies 190 and 164 seem most influential
  • 190 = Luo et al. (2019), Frontiers: Machines vs. humans: The impact of artificial intelligence chatbot disclosure on customer purchases, marketing science
  • 164 = Banker & Khetani (2019), Algorithm overdependence: How the use of algorithmic recommendation systems can increase risks to consumer well-being, Journal of Public Policy & Marketing
plot(cookdist1, type="o", pch=19, xlab="Observed effect size", ylab="Cook's Distance")

Forest Plot

  • Meta-analyses are usually illustrated using a forest plot.
  • A forest plot displays effect estimates and confidence intervals for both individual studies and meta-analyses.
  • Each study is represented by a block at the point estimate of intervention effect with a horizontal line extending either side of the block.
  • The area of the block indicates the weight assigned to that study in the meta-analysis while the horizontal line depicts the confidence interval (usually with a 95% level of confidence).
  • The area of the block and the confidence interval convey similar information, but both make different contributions to the graphic.
  • The confidence interval depicts the range of intervention effects compatible with the study’s result.
  • The size of the block draws the eye towards the studies with larger weight (usually those with narrower confidence intervals), which dominate the calculation of the summary result, presented as a diamond at the bottom.
  • If confidence intervals for the results of individual studies (generally depicted graphically using horizontal lines) have poor overlap, this generally indicates the presence of statistical heterogeneity.
## Forest Plot ####
meta_data$cohens_d_plot <- format(round(meta_data$cohens_d, 2), nsmall = 2)
meta_data$cohens_d_plot <- as.numeric(meta_data$cohens_d_plot)

meta_data$pubyear <- as.numeric(as.character(meta_data$pubyear))
forest(multilevel,
       annotate = FALSE,
       addfit = TRUE,
       addpred = FALSE,
       showweights = FALSE,
       xlim=c(-10,1), # adjust horizontal plot region limits
       ilab=cbind(meta_data$pubyear, meta_data$study_no, meta_data$n, meta_data$cohens_d_plot),
       ilab.xpos=c(-5.40, -4.80, -4.30, -3.65),
       ilab.pos = 4,
       xlab = "Standardized Mean Difference",
       order="obs",
       header = TRUE, 
       slab = meta_data$authors_short)
op <- par(cex=1, font=2)
text(x = c(-5.40, -4.80, -4.30, -3.65), y = 213, label = c("Year","No.", "N", "d"), pos = 4)

par(op)

Funnel Plot

funnel(multilevel, yaxis = "sei", main = "Funnel Plot Multi-level Random-Effects Model without Moderators", label = 5, offset=1, back = "lightgrey", pch = 20)

Formal Tests of Publication Bias

  • File drawer test assessing how many zero effects would need to exist in file drawers to make the findings non-significant. Not measuring whether publication bias exists but how strong it would need to be (if existing) to make the results of meta-analytic calculations irrelevant. Resulting value is assessed regarding realism. Can there really be this many unpublished studies? Rosenthal suggests 5k + 10 as a threshold. Cons: ignores heterogeneity between studies and effect sizes (moderators)
  • Rank test of Begg and Mazumdar (1994): If there is a publication bias, small effect sizes with high sampling variances are missing, this should lead to a significant correlation between the ranking of the effect sizes and the sampling variances.
  • Egger test: If there is a Publication bias, the standard error (or precision, sample size) has a significant effect on the effect size in a metaregression (could also be done in the full meta-regression model including many moderators)
  • Trimm-and-fill method: How many effect sizes are missing and where to make the funnel plot symmetric?

Interpretations:

  • 261326 > 1065 –> We can be pretty sure, that we have no problem with publication bias cause even if there was publication bias, the amount of unpublished work would need to be unrelaistically high to make results irrelevant.
  • The visual impression of the funnel plot is confirmed by the Rank test of Begg and Mazumdar (1994) which yields a statistically significant p-value.
  • However: Egger Test does not point toward publication bias
## Failsafe N ####
fsn(cohens_d, vi=var_d, data = meta_data, type = "Rosenthal")
## 
## Fail-safe N Calculation Using the Rosenthal Approach
## 
## Observed Significance Level: <.0001
## Target Significance Level:   0.05
## 
## Fail-safe N: 261326
# Threshold
(5*211)+10
## [1] 1065
# Rank test of Begg and Mazumdar (1994)
ranktest(multilevel)
## 
## Rank Correlation Test for Funnel Plot Asymmetry
## 
## Kendall's tau = -0.1308, p = 0.0047
# Egger Test (Regress effect size on s.e. or precision)
eggermodel_d_1 <- summary(lm(cohens_d ~ n, data = meta_data))
eggermodel_d_1
## 
## Call:
## lm(formula = cohens_d ~ n, data = meta_data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -2.1672 -0.2979 -0.0172  0.4000  1.9883 
## 
## Coefficients:
##                Estimate  Std. Error t value      Pr(>|t|)    
## (Intercept) -0.25901207  0.04213389  -6.147 0.00000000394 ***
## n           -0.00001377  0.00001888  -0.729         0.467    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.5914 on 209 degrees of freedom
## Multiple R-squared:  0.002539,   Adjusted R-squared:  -0.002234 
## F-statistic: 0.532 on 1 and 209 DF,  p-value: 0.4666

2. Including Moderators

Test of Moderators as Predictors Separately

  • Pubyear (+, *)
  • DV Scaling - Discrete (+, **)
  • DV Measurement Type - Self report (-, **)
  • Incentive Compatible - Yes (+, .)
  • US - Yes (+, **)
  • Management - Yes (+, *)
  • Subjective vs- Objective - Objective (+, *)
  • Severity - Medium (-, **)
  • Severity - High (-, ***)
  • Human Label - Expert (-, ***)
  • Algorithm Label - Yes (+. *)
  • Artificial Intelligence Label - Yes (+, .)
Publication Moderators
outlet_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0000  0.0000     99     no        study_id 
## sigma^2.2  0.1773  0.4211    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 178) = 15482.8331, p-val < .0001
## 
## Test of Moderators (coefficients 2:33):
## QM(df = 32) = 162.6074, p-val < .0001
## 
## Model Results:
## 
##                                                                                     estimate 
## intrcpt                                                                              -0.4166 
## outletAccident Analysis & Prevention                                                 -0.2146 
## outletAI & Society                                                                    0.3321 
## outletBig Data & Society                                                             -0.3368 
## outletBusiness & Information Systems Engineering                                      0.6576 
## outletCognition                                                                      -0.4928 
## outletComputers in Human Behavior                                                     0.3071 
## outletEuropean Journal of Marketing                                                   0.1585 
## outletHeliyon                                                                         0.7068 
## outletHuman Factors                                                                   0.1690 
## outletInternational Journal of Human-Computer Studies                                 0.1221 
## outletInternational Journal of Selection and Assessment                               0.1949 
## outletJournal of Accounting Research                                                 -0.2116 
## outletJournal of Behavioral Decision Making                                          -0.0281 
## outletJournal of Consumer Psychology                                                 -0.2382 
## outletJournal of Consumer Research                                                   -0.2269 
## outletJournal of Crime and Justice                                                    0.3092 
## outletJournal of Experimental Psychology: Applied                                     0.7356 
## outletJournal of Forecasting                                                          0.4886 
## outletJournal of Management Accounting Research                                       0.1059 
## outletJournal of Marketing                                                            0.2949 
## outletJournal of Marketing Research                                                   0.0522 
## outletJournal of Public Policy & Marketing                                            1.7128 
## outletMarketing Science                                                              -0.3138 
## outletMedical Decision Making                                                         0.3353 
## outletNature Human Behavior                                                           0.1616 
## outletNeuron                                                                         -0.1579 
## outletOrganizational Behavior and Human Decision Processes                            0.8291 
## outletPLOS ONE                                                                       -0.1303 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems    0.4166 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society          0.4624 
## outletProceedings of the ACM on Human-Computer Interaction                            0.4369 
## outletTechnological Forecasting and Social Change                                     0.8388 
##                                                                                         se 
## intrcpt                                                                             0.1286 
## outletAccident Analysis & Prevention                                                0.2166 
## outletAI & Society                                                                  0.1907 
## outletBig Data & Society                                                            0.1834 
## outletBusiness & Information Systems Engineering                                    0.4480 
## outletCognition                                                                     0.1873 
## outletComputers in Human Behavior                                                   0.1858 
## outletEuropean Journal of Marketing                                                 0.2811 
## outletHeliyon                                                                       0.2806 
## outletHuman Factors                                                                 0.2358 
## outletInternational Journal of Human-Computer Studies                               0.2497 
## outletInternational Journal of Selection and Assessment                             0.1700 
## outletJournal of Accounting Research                                                0.2794 
## outletJournal of Behavioral Decision Making                                         0.1564 
## outletJournal of Consumer Psychology                                                0.3253 
## outletJournal of Consumer Research                                                  0.1784 
## outletJournal of Crime and Justice                                                  0.2475 
## outletJournal of Experimental Psychology: Applied                                   0.1920 
## outletJournal of Forecasting                                                        0.2789 
## outletJournal of Management Accounting Research                                     0.2779 
## outletJournal of Marketing                                                          0.2098 
## outletJournal of Marketing Research                                                 0.2474 
## outletJournal of Public Policy & Marketing                                          0.3319 
## outletMarketing Science                                                             0.2469 
## outletMedical Decision Making                                                       0.1932 
## outletNature Human Behavior                                                         0.2480 
## outletNeuron                                                                        0.3585 
## outletOrganizational Behavior and Human Decision Processes                          0.1863 
## outletPLOS ONE                                                                      0.2761 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems  0.4432 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society        0.3286 
## outletProceedings of the ACM on Human-Computer Interaction                          0.1867 
## outletTechnological Forecasting and Social Change                                   0.2794 
##                                                                                        zval 
## intrcpt                                                                             -3.2394 
## outletAccident Analysis & Prevention                                                -0.9905 
## outletAI & Society                                                                   1.7415 
## outletBig Data & Society                                                            -1.8365 
## outletBusiness & Information Systems Engineering                                     1.4678 
## outletCognition                                                                     -2.6311 
## outletComputers in Human Behavior                                                    1.6535 
## outletEuropean Journal of Marketing                                                  0.5639 
## outletHeliyon                                                                        2.5187 
## outletHuman Factors                                                                  0.7166 
## outletInternational Journal of Human-Computer Studies                                0.4889 
## outletInternational Journal of Selection and Assessment                              1.1470 
## outletJournal of Accounting Research                                                -0.7573 
## outletJournal of Behavioral Decision Making                                         -0.1794 
## outletJournal of Consumer Psychology                                                -0.7322 
## outletJournal of Consumer Research                                                  -1.2716 
## outletJournal of Crime and Justice                                                   1.2494 
## outletJournal of Experimental Psychology: Applied                                    3.8314 
## outletJournal of Forecasting                                                         1.7518 
## outletJournal of Management Accounting Research                                      0.3811 
## outletJournal of Marketing                                                           1.4054 
## outletJournal of Marketing Research                                                  0.2111 
## outletJournal of Public Policy & Marketing                                           5.1601 
## outletMarketing Science                                                             -1.2713 
## outletMedical Decision Making                                                        1.7351 
## outletNature Human Behavior                                                          0.6516 
## outletNeuron                                                                        -0.4406 
## outletOrganizational Behavior and Human Decision Processes                           4.4497 
## outletPLOS ONE                                                                      -0.4721 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems   0.9398 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society         1.4071 
## outletProceedings of the ACM on Human-Computer Interaction                           2.3408 
## outletTechnological Forecasting and Social Change                                    3.0021 
##                                                                                       pval 
## intrcpt                                                                             0.0012 
## outletAccident Analysis & Prevention                                                0.3219 
## outletAI & Society                                                                  0.0816 
## outletBig Data & Society                                                            0.0663 
## outletBusiness & Information Systems Engineering                                    0.1422 
## outletCognition                                                                     0.0085 
## outletComputers in Human Behavior                                                   0.0982 
## outletEuropean Journal of Marketing                                                 0.5728 
## outletHeliyon                                                                       0.0118 
## outletHuman Factors                                                                 0.4736 
## outletInternational Journal of Human-Computer Studies                               0.6249 
## outletInternational Journal of Selection and Assessment                             0.2514 
## outletJournal of Accounting Research                                                0.4489 
## outletJournal of Behavioral Decision Making                                         0.8576 
## outletJournal of Consumer Psychology                                                0.4640 
## outletJournal of Consumer Research                                                  0.2035 
## outletJournal of Crime and Justice                                                  0.2115 
## outletJournal of Experimental Psychology: Applied                                   0.0001 
## outletJournal of Forecasting                                                        0.0798 
## outletJournal of Management Accounting Research                                     0.7031 
## outletJournal of Marketing                                                          0.1599 
## outletJournal of Marketing Research                                                 0.8328 
## outletJournal of Public Policy & Marketing                                          <.0001 
## outletMarketing Science                                                             0.2036 
## outletMedical Decision Making                                                       0.0827 
## outletNature Human Behavior                                                         0.5146 
## outletNeuron                                                                        0.6595 
## outletOrganizational Behavior and Human Decision Processes                          <.0001 
## outletPLOS ONE                                                                      0.6369 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems  0.3473 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society        0.1594 
## outletProceedings of the ACM on Human-Computer Interaction                          0.0192 
## outletTechnological Forecasting and Social Change                                   0.0027 
##                                                                                       ci.lb 
## intrcpt                                                                             -0.6686 
## outletAccident Analysis & Prevention                                                -0.6391 
## outletAI & Society                                                                  -0.0416 
## outletBig Data & Society                                                            -0.6962 
## outletBusiness & Information Systems Engineering                                    -0.2205 
## outletCognition                                                                     -0.8599 
## outletComputers in Human Behavior                                                   -0.0569 
## outletEuropean Journal of Marketing                                                 -0.3924 
## outletHeliyon                                                                        0.1568 
## outletHuman Factors                                                                 -0.2932 
## outletInternational Journal of Human-Computer Studies                               -0.3673 
## outletInternational Journal of Selection and Assessment                             -0.1382 
## outletJournal of Accounting Research                                                -0.7593 
## outletJournal of Behavioral Decision Making                                         -0.3347 
## outletJournal of Consumer Psychology                                                -0.8758 
## outletJournal of Consumer Research                                                  -0.5766 
## outletJournal of Crime and Justice                                                  -0.1759 
## outletJournal of Experimental Psychology: Applied                                    0.3593 
## outletJournal of Forecasting                                                        -0.0581 
## outletJournal of Management Accounting Research                                     -0.4387 
## outletJournal of Marketing                                                          -0.1164 
## outletJournal of Marketing Research                                                 -0.4327 
## outletJournal of Public Policy & Marketing                                           1.0622 
## outletMarketing Science                                                             -0.7976 
## outletMedical Decision Making                                                       -0.0434 
## outletNature Human Behavior                                                         -0.3245 
## outletNeuron                                                                        -0.8605 
## outletOrganizational Behavior and Human Decision Processes                           0.4639 
## outletPLOS ONE                                                                      -0.6714 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems  -0.4521 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society        -0.1817 
## outletProceedings of the ACM on Human-Computer Interaction                           0.0711 
## outletTechnological Forecasting and Social Change                                    0.2912 
##                                                                                       ci.ub 
## intrcpt                                                                             -0.1645 
## outletAccident Analysis & Prevention                                                 0.2100 
## outletAI & Society                                                                   0.7058 
## outletBig Data & Society                                                             0.0226 
## outletBusiness & Information Systems Engineering                                     1.5356 
## outletCognition                                                                     -0.1257 
## outletComputers in Human Behavior                                                    0.6712 
## outletEuropean Journal of Marketing                                                  0.7095 
## outletHeliyon                                                                        1.2567 
## outletHuman Factors                                                                  0.6311 
## outletInternational Journal of Human-Computer Studies                                0.6115 
## outletInternational Journal of Selection and Assessment                              0.5281 
## outletJournal of Accounting Research                                                 0.3361 
## outletJournal of Behavioral Decision Making                                          0.2785 
## outletJournal of Consumer Psychology                                                 0.3994 
## outletJournal of Consumer Research                                                   0.1228 
## outletJournal of Crime and Justice                                                   0.7943 
## outletJournal of Experimental Psychology: Applied                                    1.1119 
## outletJournal of Forecasting                                                         1.0352 
## outletJournal of Management Accounting Research                                      0.6505 
## outletJournal of Marketing                                                           0.7061 
## outletJournal of Marketing Research                                                  0.5372 
## outletJournal of Public Policy & Marketing                                           2.3634 
## outletMarketing Science                                                              0.1700 
## outletMedical Decision Making                                                        0.7140 
## outletNature Human Behavior                                                          0.6478 
## outletNeuron                                                                         0.5447 
## outletOrganizational Behavior and Human Decision Processes                           1.1943 
## outletPLOS ONE                                                                       0.4108 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems   1.2852 
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society         1.1065 
## outletProceedings of the ACM on Human-Computer Interaction                           0.8027 
## outletTechnological Forecasting and Social Change                                    1.3865 
##  
## intrcpt                                                                              ** 
## outletAccident Analysis & Prevention 
## outletAI & Society                                                                    . 
## outletBig Data & Society                                                              . 
## outletBusiness & Information Systems Engineering 
## outletCognition                                                                      ** 
## outletComputers in Human Behavior                                                     . 
## outletEuropean Journal of Marketing 
## outletHeliyon                                                                         * 
## outletHuman Factors 
## outletInternational Journal of Human-Computer Studies 
## outletInternational Journal of Selection and Assessment 
## outletJournal of Accounting Research 
## outletJournal of Behavioral Decision Making 
## outletJournal of Consumer Psychology 
## outletJournal of Consumer Research 
## outletJournal of Crime and Justice 
## outletJournal of Experimental Psychology: Applied                                   *** 
## outletJournal of Forecasting                                                          . 
## outletJournal of Management Accounting Research 
## outletJournal of Marketing 
## outletJournal of Marketing Research 
## outletJournal of Public Policy & Marketing                                          *** 
## outletMarketing Science 
## outletMedical Decision Making                                                         . 
## outletNature Human Behavior 
## outletNeuron 
## outletOrganizational Behavior and Human Decision Processes                          *** 
## outletPLOS ONE 
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems      
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society            
## outletProceedings of the ACM on Human-Computer Interaction                            * 
## outletTechnological Forecasting and Social Change                                    ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
discipline_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0631  0.2512     99     no        study_id 
## sigma^2.2  0.2475  0.4975    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 203) = 19428.7658, p-val < .0001
## 
## Test of Moderators (coefficients 2:8):
## QM(df = 7) = 10.1948, p-val = 0.1778
## 
## Model Results:
## 
##                                estimate      se     zval    pval    ci.lb 
## intrcpt                         -0.2229  0.1926  -1.1576  0.2470  -0.6004 
## disciplineInformation Systems    0.0614  0.2183   0.2812  0.7785  -0.3665 
## disciplineLaw                    0.1156  0.4031   0.2869  0.7742  -0.6744 
## disciplineManagement             0.1801  0.2185   0.8243  0.4098  -0.2482 
## disciplineMarketing             -0.1374  0.2189  -0.6279  0.5300  -0.5664 
## disciplineMedicine               0.0479  0.2789   0.1716  0.8637  -0.4988 
## disciplinePsychology            -0.1481  0.2145  -0.6905  0.4899  -0.5685 
## disciplineTransportation        -0.3880  0.3047  -1.2735  0.2028  -0.9851 
##                                 ci.ub 
## intrcpt                        0.1545    
## disciplineInformation Systems  0.4893    
## disciplineLaw                  0.9056    
## disciplineManagement           0.6084    
## disciplineMarketing            0.2915    
## disciplineMedicine             0.5946    
## disciplinePsychology           0.2723    
## disciplineTransportation       0.2091    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
pubyear_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0655  0.2559     99     no        study_id 
## sigma^2.2  0.2539  0.5039    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20738.2794, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.6765, p-val = 0.0306
## 
## Model Results:
## 
##          estimate       se     zval    pval     ci.lb    ci.ub 
## intrcpt  -46.3067  21.2990  -2.1741  0.0297  -88.0520  -4.5615  * 
## pubyear    0.0228   0.0106   2.1625  0.0306    0.0021   0.0435  * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
pubtype_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0698  0.2642     99     no        study_id 
## sigma^2.2  0.2531  0.5031    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 208) = 20692.6921, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 3.2342, p-val = 0.1985
## 
## Model Results:
## 
##                         estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt                   0.0444  0.1884   0.2354  0.8139  -0.3249  0.4136    
## pubtypeJournal article   -0.3011  0.1950  -1.5443  0.1225  -0.6833  0.0811    
## pubtypeWorking Paper     -0.4769  0.2798  -1.7046  0.0883  -1.0253  0.0714  . 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
published_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0767  0.2769     99     no        study_id 
## sigma^2.2  0.2529  0.5029    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20927.4858, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.3118, p-val = 0.5766
## 
## Model Results:
## 
##               estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt        -0.1718  0.1422  -1.2080  0.2271  -0.4505  0.1070    
## publishedyes   -0.0844  0.1511  -0.5584  0.5766  -0.3806  0.2118    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Method Moderators
within_subject_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0769  0.2773     99     no        study_id 
## sigma^2.2  0.2528  0.5028    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20928.2761, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.2925, p-val = 0.5886
## 
## Model Results:
## 
##                    estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt             -0.2359  0.0520  -4.5406  <.0001  -0.3378  -0.1341  *** 
## within_subjectyes   -0.0748  0.1384  -0.5409  0.5886  -0.3461   0.1964      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
# random_model
field_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0729  0.2699     99     no        study_id 
## sigma^2.2  0.2537  0.5037    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20315.2533, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 1.4417, p-val = 0.2299
## 
## Model Results:
## 
##           estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt    -0.2325  0.0492  -4.7288  <.0001  -0.3289  -0.1361  *** 
## fieldyes   -0.2440  0.2032  -1.2007  0.2299  -0.6422   0.1543      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_cat_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0806  0.2840     99     no        study_id 
## sigma^2.2  0.2501  0.5001    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 208) = 20279.7643, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 0.4593, p-val = 0.7948
## 
## Model Results:
## 
##                              estimate      se     zval    pval    ci.lb 
## intrcpt                       -0.2700  0.1083  -2.4924  0.0127  -0.4823 
## dv_catreal outcome            -0.0437  0.1673  -0.2609  0.7941  -0.3716 
## dv_catsubjective perception    0.0419  0.1181   0.3548  0.7227  -0.1896 
##                                ci.ub 
## intrcpt                      -0.0577  * 
## dv_catreal outcome            0.2843    
## dv_catsubjective perception   0.2734    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
subjective_perception_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0816  0.2857     99     no        study_id 
## sigma^2.2  0.2495  0.4995    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20322.2147, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.4231, p-val = 0.5154
## 
## Model Results:
## 
##                           estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt                    -0.2892  0.0821  -3.5228  0.0004  -0.4501  -0.1283 
## subjective_perceptionyes    0.0617  0.0949   0.6505  0.5154  -0.1242   0.2477 
##  
## intrcpt                   *** 
## subjective_perceptionyes 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_scaling_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0767  0.2769     99     no        study_id 
## sigma^2.2  0.2431  0.4930    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20689.3842, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 6.9632, p-val = 0.0083
## 
## Model Results:
## 
##                     estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt              -0.2801  0.0493  -5.6830  <.0001  -0.3767  -0.1835  *** 
## dv_scalingdiscrete    0.4606  0.1746   2.6388  0.0083   0.1185   0.8027   ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_mt_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0746  0.2732     99     no        study_id 
## sigma^2.2  0.2427  0.4927    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20759.4199, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 7.9620, p-val = 0.0048
## 
## Model Results:
## 
##                                 estimate      se     zval    pval    ci.lb 
## intrcpt                           0.0145  0.1038   0.1393  0.8892  -0.1891 
## dv_measurement_typeself-report   -0.3231  0.1145  -2.8217  0.0048  -0.5476 
##                                   ci.ub 
## intrcpt                          0.2180     
## dv_measurement_typeself-report  -0.0987  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
compensation_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0770  0.2775     99     no        study_id 
## sigma^2.2  0.2521  0.5021    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20927.6131, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.6704, p-val = 0.4129
## 
## Model Results:
## 
##                  estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt           -0.3206  0.1026  -3.1259  0.0018  -0.5217  -0.1196  ** 
## compensationyes    0.0951  0.1162   0.8188  0.4129  -0.1326   0.3228     
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
incentive_compatible_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0719  0.2682     99     no        study_id 
## sigma^2.2  0.2521  0.5021    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20881.3933, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 2.8262, p-val = 0.0927
## 
## Model Results:
## 
##                          estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt                   -0.2735  0.0501  -5.4607  <.0001  -0.3717  -0.1753 
## incentive_compatibleyes    0.2573  0.1530   1.6811  0.0927  -0.0427   0.5573 
##  
## intrcpt                  *** 
## incentive_compatibleyes    . 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
preregistered_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0792  0.2815     99     no        study_id 
## sigma^2.2  0.2516  0.5016    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20928.2224, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.0247, p-val = 0.8751
## 
## Model Results:
## 
##                   estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt            -0.2421  0.0554  -4.3739  <.0001  -0.3506  -0.1336  *** 
## preregisteredyes   -0.0179  0.1140  -0.1572  0.8751  -0.2413   0.2054      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Sample Moderators
m_age_model
## 
## Multivariate Meta-Analysis Model (k = 179; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.1156  0.3401     83     no        study_id 
## sigma^2.2  0.2367  0.4865    179     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 177) = 16691.9554, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.9053, p-val = 0.3414
## 
## Model Results:
## 
##          estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt   -0.5064  0.2867  -1.7660  0.0774  -1.0684  0.0556  . 
## m_age      0.0078  0.0082   0.9515  0.3414  -0.0082  0.0238    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
percntg_females_model
## 
## Multivariate Meta-Analysis Model (k = 193; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.1191  0.3451     89     no        study_id 
## sigma^2.2  0.2179  0.4668    193     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 191) = 19982.4838, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.1909, p-val = 0.6621
## 
## Model Results:
## 
##                  estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt           -0.3409  0.2305  -1.4792  0.1391  -0.7927  0.1108    
## percntg_females    0.0019  0.0043   0.4370  0.6621  -0.0065  0.0102    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
online_model
## 
## Multivariate Meta-Analysis Model (k = 206; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0747  0.2734     95     no        study_id 
## sigma^2.2  0.2518  0.5018    206     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 204) = 20455.1362, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.0662, p-val = 0.7970
## 
## Model Results:
## 
##            estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt     -0.2470  0.0708  -3.4870  0.0005  -0.3858  -0.1082  *** 
## onlineyes   -0.0250  0.0974  -0.2572  0.7970  -0.2159   0.1658      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
students_model
## 
## Multivariate Meta-Analysis Model (k = 206; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0722  0.2687     95     no        study_id 
## sigma^2.2  0.2531  0.5030    206     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 204) = 20507.4422, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.3251, p-val = 0.5685
## 
## Model Results:
## 
##              estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt       -0.2455  0.0549  -4.4727  <.0001  -0.3531  -0.1379  *** 
## studentsyes   -0.0661  0.1158  -0.5702  0.5685  -0.2931   0.1610      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
location_model
## 
## Multivariate Meta-Analysis Model (k = 142; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0827  0.2875     50     no        study_id 
## sigma^2.2  0.2329  0.4826    142     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 131) = 13034.9118, p-val < .0001
## 
## Test of Moderators (coefficients 2:11):
## QM(df = 10) = 23.4366, p-val = 0.0092
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt                 -0.3904  0.2027  -1.9262  0.0541  -0.7876  0.0069  . 
## locationAsia            -0.3402  0.4267  -0.7975  0.4252  -1.1765  0.4960    
## locationCanada          -0.2775  0.2990  -0.9279  0.3535  -0.8635  0.3086    
## locationChina           -0.2156  0.3138  -0.6872  0.4920  -0.8306  0.3994    
## locationGermany          0.2399  0.4957   0.4841  0.6283  -0.7316  1.2115    
## locationItaly           -0.4084  0.4660  -0.8764  0.3808  -1.3217  0.5049    
## locationmultiple        -0.3001  0.2614  -1.1480  0.2510  -0.8125  0.2123    
## locationNetherlands      0.3059  0.3870   0.7904  0.4293  -0.4526  1.0643    
## locationNon-Us           0.6035  0.3143   1.9198  0.0549  -0.0126  1.2196  . 
## locationTurkey          -0.2044  0.4545  -0.4497  0.6530  -1.0952  0.6865    
## locationUnited States    0.3182  0.2218   1.4344  0.1515  -0.1166  0.7529    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
US_model
## 
## Multivariate Meta-Analysis Model (k = 124; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0595  0.2440     41     no        study_id 
## sigma^2.2  0.2499  0.4999    124     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 122) = 11613.9151, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.3688, p-val = 0.0366
## 
## Model Results:
## 
##          estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt   -0.4185  0.1015  -4.1242  <.0001  -0.6174  -0.2196  *** 
## USyes      0.2544  0.1217   2.0902  0.0366   0.0158   0.4930    * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
sample_type_model
## 
## Multivariate Meta-Analysis Model (k = 206; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0716  0.2676     95     no        study_id 
## sigma^2.2  0.2527  0.5027    206     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 203) = 20455.1057, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 0.8274, p-val = 0.6612
## 
## Model Results:
## 
##                      estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt               -0.2720  0.0664  -4.0990  <.0001  -0.4020  -0.1419  *** 
## sample_typeothers      0.0831  0.1176   0.7072  0.4795  -0.1473   0.3136      
## sample_typestudents   -0.0397  0.1215  -0.3264  0.7441  -0.2778   0.1985      
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Domain Moderator
domain_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0666  0.2581     99     no        study_id 
## sigma^2.2  0.2070  0.4550    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 196) = 16979.2065, p-val < .0001
## 
## Test of Moderators (coefficients 2:15):
## QM(df = 14) = 41.6825, p-val = 0.0001
## 
## Model Results:
## 
##                           estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt                    -0.0474  0.3339  -0.1419  0.8872  -0.7019   0.6071 
## domainart                  -0.2689  0.4518  -0.5953  0.5516  -1.1544   0.6165 
## domainautonomous driving   -0.7079  0.3922  -1.8051  0.0711  -1.4765   0.0607 
## domainbaking/cooking       -0.2926  0.4697  -0.6230  0.5333  -1.2131   0.6279 
## domainfinance              -0.4281  0.3854  -1.1107  0.2667  -1.1834   0.3273 
## domainlaw                  -0.1957  0.3713  -0.5270  0.5982  -0.9233   0.5320 
## domainmanagement           -0.0030  0.3468  -0.0087  0.9930  -0.6827   0.6767 
## domainmedicine             -0.3805  0.3463  -1.0987  0.2719  -1.0592   0.2983 
## domainmilitary             -1.2198  0.4534  -2.6902  0.0071  -2.1086  -0.3311 
## domainmultiple             -0.1897  0.3525  -0.5382  0.5904  -0.8806   0.5012 
## domainnews                 -0.0455  0.4557  -0.0999  0.9204  -0.9387   0.8476 
## domainother                 0.1107  0.3894   0.2843  0.7762  -0.6526   0.8740 
## domainreal estate          -0.6039  0.5207  -1.1598  0.2461  -1.6244   0.4166 
## domainshopping              0.3287  0.3898   0.8433  0.3990  -0.4352   1.0926 
## domainsocial                0.0589  0.3809   0.1546  0.8772  -0.6876   0.8054 
##  
## intrcpt 
## domainart 
## domainautonomous driving   . 
## domainbaking/cooking 
## domainfinance 
## domainlaw 
## domainmanagement 
## domainmedicine 
## domainmilitary            ** 
## domainmultiple 
## domainnews 
## domainother 
## domainreal estate 
## domainshopping 
## domainsocial 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
management_model
## 
## Multivariate Meta-Analysis Model (k = 205; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0954  0.3089     95     no        study_id 
## sigma^2.2  0.2330  0.4827    205     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 203) = 20665.1439, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 9.7347, p-val = 0.0018
## 
## Model Results:
## 
##                estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt         -0.3444  0.0592  -5.8215  <.0001  -0.4603  -0.2284  *** 
## managementyes    0.3485  0.1117   3.1201  0.0018   0.1296   0.5674   ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
subj_vs_obj_model
## 
## Multivariate Meta-Analysis Model (k = 187; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0458  0.2141     92     no        study_id 
## sigma^2.2  0.2852  0.5340    187     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 185) = 19218.5561, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.1856, p-val = 0.0408
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt                 -0.2899  0.0538  -5.3913  <.0001  -0.3953  -0.1845  *** 
## subj_vs_objsubjective    0.2382  0.1164   2.0459  0.0408   0.0100   0.4664    * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
severity_model
## 
## Multivariate Meta-Analysis Model (k = 176; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0737  0.2715     89     no        study_id 
## sigma^2.2  0.2577  0.5076    176     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 173) = 19100.0392, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 20.3878, p-val < .0001
## 
## Model Results:
## 
##                 estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt           0.0947  0.1031   0.9187  0.3583  -0.1074   0.2968      
## severitymedium   -0.3376  0.1269  -2.6613  0.0078  -0.5863  -0.0890   ** 
## severityhigh     -0.5959  0.1320  -4.5151  <.0001  -0.8545  -0.3372  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Stimuli Moderators
human_type_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0445  0.2110     99     no        study_id 
## sigma^2.2  0.2410  0.4910    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 204) = 20015.5297, p-val < .0001
## 
## Test of Moderators (coefficients 2:7):
## QM(df = 6) = 26.0937, p-val = 0.0002
## 
## Model Results:
## 
##                                     estimate      se     zval    pval    ci.lb 
## intrcpt                              -0.2207  0.1456  -1.5162  0.1295  -0.5061 
## human_typeexpert(s)                  -0.1273  0.1534  -0.8295  0.4068  -0.4280 
## human_typehuman-controlled machine   -0.0376  0.3144  -0.1197  0.9047  -0.6538 
## human_typemultiple                    0.2787  0.2973   0.9374  0.3485  -0.3040 
## human_typeother                       0.0703  0.4379   0.1605  0.8725  -0.7880 
## human_typeother participant(s)        0.3605  0.2028   1.7779  0.0754  -0.0369 
## human_typeparticipants themselves     0.9534  0.3077   3.0982  0.0019   0.3503 
##                                      ci.ub 
## intrcpt                             0.0646     
## human_typeexpert(s)                 0.1735     
## human_typehuman-controlled machine  0.5785     
## human_typemultiple                  0.8613     
## human_typeother                     0.9286     
## human_typeother participant(s)      0.7580   . 
## human_typeparticipants themselves   1.5566  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
expert_model
## 
## Multivariate Meta-Analysis Model (k = 202; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0618  0.2486     92     no        study_id 
## sigma^2.2  0.2357  0.4854    202     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 200) = 20003.5319, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 5.3091, p-val = 0.0212
## 
## Model Results:
## 
##            estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt     -0.0962  0.0939  -1.0246  0.3055  -0.2801   0.0878    
## expertyes   -0.2476  0.1075  -2.3041  0.0212  -0.4583  -0.0370  * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
algorithm_dummy_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0735  0.2711     99     no        study_id 
## sigma^2.2  0.2470  0.4970    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20615.2367, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 5.4307, p-val = 0.0198
## 
## Model Results:
## 
##                     estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt              -0.3511  0.0653  -5.3789  <.0001  -0.4791  -0.2232  *** 
## algorithm_dummyyes    0.2200  0.0944   2.3304  0.0198   0.0350   0.4051    * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
ai_dummy_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0699  0.2643     99     no        study_id 
## sigma^2.2  0.2524  0.5024    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20901.3765, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 3.7717, p-val = 0.0521
## 
## Model Results:
## 
##              estimate      se     zval    pval    ci.lb    ci.ub 
## intrcpt       -0.2992  0.0544  -5.5024  <.0001  -0.4058  -0.1926  *** 
## ai_dummyyes    0.2134  0.1099   1.9421  0.0521  -0.0020   0.4288    . 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
textual_model
## 
## Multivariate Meta-Analysis Model (k = 211; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0783  0.2799     99     no        study_id 
## sigma^2.2  0.2511  0.5011    211     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 209) = 20499.3534, p-val < .0001
## 
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.7947, p-val = 0.3727
## 
## Model Results:
## 
##             estimate      se     zval    pval    ci.lb   ci.ub 
## intrcpt      -0.1503  0.1180  -1.2735  0.2028  -0.3817  0.0810    
## textualyes   -0.1153  0.1293  -0.8914  0.3727  -0.3688  0.1382    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Multi-level random-effects meta-regression model with moderators

Model Output

multilevelmodel
## 
## Multivariate Meta-Analysis Model (k = 179; method: ML)
## 
## Variance Components:
## 
##             estim    sqrt  nlvls  fixed          factor 
## sigma^2.1  0.0311  0.1764     83     no        study_id 
## sigma^2.2  0.2115  0.4599    179     no  study_id/es_id 
## 
## Test for Residual Heterogeneity:
## QE(df = 166) = 14236.0668, p-val < .0001
## 
## Test of Moderators (coefficients 2:13):
## QM(df = 12) = 63.0851, p-val < .0001
## 
## Model Results:
## 
##                      estimate       se     zval    pval      ci.lb     ci.ub 
## intrcpt              -93.0234  36.0786  -2.5784  0.0099  -163.7362  -22.3105 
## pubyear                0.0454   0.0179   2.5399  0.0111     0.0104    0.0804 
## within_subject         0.0876   0.0786   1.1152  0.2647    -0.0664    0.2417 
## dv_measurement_type    0.1599   0.0712   2.2461  0.0247     0.0204    0.2995 
## preregistered         -0.0011   0.0602  -0.0190  0.9849    -0.1191    0.1168 
## m_age                  0.0234   0.0075   3.1226  0.0018     0.0087    0.0381 
## percntg_females        0.0151   0.0042   3.6240  0.0003     0.0069    0.0233 
## online                -0.1571   0.0588  -2.6715  0.0076    -0.2723   -0.0418 
## US                     0.0728   0.0631   1.1536  0.2487    -0.0509    0.1966 
## management             0.2627   0.0555   4.7339  <.0001     0.1539    0.3714 
## expert                -0.2215   0.0662  -3.3474  0.0008    -0.3512   -0.0918 
## ai_dummy               0.0433   0.0600   0.7209  0.4710    -0.0744    0.1609 
## textual                0.0053   0.0676   0.0788  0.9372    -0.1273    0.1379 
##  
## intrcpt               ** 
## pubyear                * 
## within_subject 
## dv_measurement_type    * 
## preregistered 
## m_age                 ** 
## percntg_females      *** 
## online                ** 
## US 
## management           *** 
## expert               *** 
## ai_dummy 
## textual 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
funnel(multilevelmodel, yaxis = "sei", main = "Funnel Plot Multi-level Random-Effects Model Including Moderators", label = 5, offset=1, back = "lightgrey", pch = 20)

Predictor Correlations

  • Tammo Bijmolt (in his slides) on Multicollinearity between moderators: “Natural empirical design”: Many moderators show very high multicollinearity. Combine levels / moderators or drop them.This is due to natural co-occurrence of design factors: researcher conducting a meta-analysis can do little about it. Use regular statistics like VIF, GVIF
  • From Abraham Paper: As there is no direct test for multicollinearity in HLMs, we identified variables that were correlated with |r| > .5 (Eisend 2014).
cordata <- select(meta_data, pubyear, within_subject, dv_measurement_type, preregistered, m_age, percntg_females, online, US, management, expert, ai_dummy, textual)
head(cordata)
##   pubyear within_subject dv_measurement_type preregistered m_age
## 1    2019             -1                  -1            -1  36.0
## 2    2019             -1                   1            -1    NA
## 3    2019             -1                   1            -1    NA
## 4    2019             -1                  -1            -1  19.6
## 5    2019             -1                  -1            -1  33.6
## 6    2019             -1                  -1            -1  36.9
##   percntg_females online US management expert ai_dummy textual
## 1            45.0      1  0          0      1       -1       1
## 2            40.0     -1  0         -1      1       -1      -1
## 3            40.0     -1  0         -1      1       -1      -1
## 4            54.9     -1  1         -1      1       -1       1
## 5            55.3      1  0         -1      1        1       1
## 6            48.1      1  0         -1      1       -1       1
res <- cor(cordata, use = "complete.obs")
round(res, 2)
##                     pubyear within_subject dv_measurement_type preregistered
## pubyear                1.00          -0.42                0.18          0.29
## within_subject        -0.42           1.00                0.22          0.14
## dv_measurement_type    0.18           0.22                1.00          0.30
## preregistered          0.29           0.14                0.30          1.00
## m_age                  0.19          -0.23               -0.14          0.16
## percntg_females       -0.27           0.09               -0.34         -0.18
## online                 0.50          -0.15                0.03          0.36
## US                     0.35          -0.09                0.11          0.23
## management            -0.08          -0.24               -0.05         -0.27
## expert                -0.09          -0.04               -0.31         -0.14
## ai_dummy               0.35          -0.08                0.13         -0.06
## textual               -0.16          -0.30               -0.47         -0.36
##                     m_age percntg_females online    US management expert
## pubyear              0.19           -0.27   0.50  0.35      -0.08  -0.09
## within_subject      -0.23            0.09  -0.15 -0.09      -0.24  -0.04
## dv_measurement_type -0.14           -0.34   0.03  0.11      -0.05  -0.31
## preregistered        0.16           -0.18   0.36  0.23      -0.27  -0.14
## m_age                1.00           -0.25   0.28  0.05       0.12   0.27
## percntg_females     -0.25            1.00  -0.23 -0.19      -0.29   0.04
## online               0.28           -0.23   1.00  0.42       0.04  -0.07
## US                   0.05           -0.19   0.42  1.00      -0.02  -0.15
## management           0.12           -0.29   0.04 -0.02       1.00   0.35
## expert               0.27            0.04  -0.07 -0.15       0.35   1.00
## ai_dummy             0.16           -0.27  -0.04  0.03      -0.16  -0.32
## textual             -0.02            0.32  -0.12 -0.15       0.18   0.26
##                     ai_dummy textual
## pubyear                 0.35   -0.16
## within_subject         -0.08   -0.30
## dv_measurement_type     0.13   -0.47
## preregistered          -0.06   -0.36
## m_age                   0.16   -0.02
## percntg_females        -0.27    0.32
## online                 -0.04   -0.12
## US                      0.03   -0.15
## management             -0.16    0.18
## expert                 -0.32    0.26
## ai_dummy                1.00   -0.08
## textual                -0.08    1.00
library(corrplot)
## corrplot 0.92 loaded
corrplot(res, type = "upper", order = "hclust", 
         tl.col = "black", tl.srt = 45)

library("PerformanceAnalytics")
## Loading required package: xts
## Loading required package: zoo
## 
## Attaching package: 'zoo'
## The following objects are masked from 'package:base':
## 
##     as.Date, as.Date.numeric
## 
## Attaching package: 'xts'
## The following objects are masked from 'package:dplyr':
## 
##     first, last
## 
## Attaching package: 'PerformanceAnalytics'
## The following object is masked from 'package:graphics':
## 
##     legend
chart.Correlation(cordata, histogram=TRUE, pch=19)

Outlier Detection

# Standardized residuals (observed residuals divided by the corresponding standard errors)
standresid <- residuals(multilevel, type="rstandard")

# Cook's distance D for each observed effect size (change in beta if one observation is dropped)
# Thresholds: D > 0.5 and > 1.0; D > 3*mean; D > 4/n; D > chisq(# regrcoeff, 0.50); any extreme D-value
cookdist <- cooks.distance(multilevel, progbar = TRUE, reestimate=FALSE)